Get Master Flow Execution Progress

This example demonstrates how to update data flow objects and execute flows in Pyramid.

The example uses API authentication driven from JavaScript. See Authentication APIs for alternatives.

// URL of the Pyramid installation and the path to the API 3.0 REST methods
var pyramidURL = "http://mySite.com/api3/";
const DEMO_FILE = 'url/test.pie';
const FILE_NAME = 'test';
/*
example  - import master flow content, fix sources and execute
	createNewFolder - f1
	findRole  - role1
	addRoleToFolder - role1 to folder of item 1
	importContent 
	validateMasterFlow
	create server - optional 
	updateSourceNodeConnection
	updateTargetNodeConnection
	updateVariableConnection
	executeMasterFlow
	getMasterFlowProgressUpdate
*/


const token = callApi(
	'authentication/authenticateUser',
		{
			userName: 'pyramid user name',
				password: 'password',
		},
		"",
			false
	);
log('got token ' + token);

const defaultTenant = callApi('access/getDefaultTenant', "",token).data;

const tenantPublicFolder = callApi('content/getPublicOrGroupFolderByTenantId', {
	validRootFolderType: 1, //the public folder of the default tenant
	tenantId: defaultTenant,
},token).data;

const folderCreation = callApi('content/createNewFolder', {
	parentFolderId: tenantPublicFolder.id,
	folderName: 'new folder',
},token).data;
const folderId = folderCreation.modifiedList[0].id;

const findRole = callApi('access/getRolesByName', {
	searchValue: 'role 1',
	searchMatchType: 2, //SearchMatchType.Equals
},token);
const roleId = findRole.data[0].roleId;
log('found role with id= ' + roleId);

//adding the role we found to the folder we created (folderId)
	roleToItemApiData: {
	itemId: folderId,
	roleId: roleId, //SearchMatchType.Equals
	},token);

const pieFile = readPieFile(DEMO_FILE);
				
const importContent = callApi('content/importContent', {
	rootFolderId: folderId,
	fileZippedData: pieFile,
	clashDefaultOption: 1, //ClashDefaultOption.REPLACE_FILE like default
	rolesAssignmentType: 3, //RoleAssignmentType.ForceParentFolderRoles take roles from parent folder
	},token).data;
log('import file completed');

//search for the item we just imported
	const findContentItem = callApi('content/findContentItem', {
		searchString: FILE_NAME,
		filterTypes: [4], //ContentTypeObject.ETLFlow
		searchMatchType: 2, //SearchMatchType.Equals
		searchRootFolderType: 1, //SearchRootFolderType.Public
	},token).data;
				
const itemId = findContentItem[0].id;
log('found the file by name: item ID is ' + itemId);

//validate master flow
const validationResult = callApi('dataSources/validateMasterFlow', {
	itemId: itemId
},token).data;

// random GUID
const serverId = '7385130a-d87f-404d-816b-5c296c27b4ad';
				
// handle sources issues
if (validationResult.sources.length > 0) {
log('create server');
				
// create server (if has alreafy get it by server name)
	const createResult = callApi('dataSources/createDataServers', [
		{
		id: serverId,
		serverName: 'apiMashwinds',
		serverType: 5,
		serverIp: '172.29.3.219',
		port: 1433,
		writeCapable: 1,
		securedByUser: false,
		serverAuthenticationMethod: 0,
		userName: 'sa',
		password: 'snap3535!',
		useGlobalAccount: false,
		overlayPyramidSecurity: false,
		},
	],token);

log('fixing sources');
				
// In this example only the server ID is replaced, where the database remains the same
const sourceObject = validationResult.sources[0];
				
const modifiedItemsResult = callApi('dataSources/updateSourceNodeConnection', {
	dataFlowNodeId: sourceObject.dataFlowNodeId,
	serverId: serverId,
	databaseName: sourceObject.databaseName,
},token).data;

log('fixing targets');
if (validationResult.targets.length > 0) {
		const targetObject = validationResult.targets[0];
		const res = callApi('dataSources/updateTargetNodeConnection', {
			useExistingDatabase: false,
			itemId: itemId,
			dataFlowNodeId: targetObject.dataFlowNodeId,
			serverId: serverId,
			databaseName: 'NewDbExample',
	},token).data;
}

log('fixing variables');
if (validationResult.variables.length > 0) {
	const variableObject = validationResult.variables[0];
		const res = callApi('dataSources/updateVariableConnection', {
			itemId: itemId,
			variableName: variableObject.variableName,
			serverId: serverId,
	},token).data;
}

log('execute master flow');
const executeMasterFlowResult = callApi('dataSources/executeMasterFlow', {
	itemId: itemId,
},token).data;

const scheduleId = executeMasterFlowResult.scheduleId;
let inProgress = true;
let counter = 0;

// while
while (inProgress && counter < 60) {
	const masterFlowProgressUpdate = callApi('dataSources/getMasterFlowProgressUpdate', scheduleId, token).data;
const taskStatus = masterFlowProgressUpdate.taskStatus;
if (
	taskStatus == null ||
	taskStatus == 0 /*TaskStatus.PENDING*/ ||
	taskStatus == 4 /*TaskStatus.RUNNING*/
) {
counter++;
	setTimeout(() => console.log('sleep'), 1000);
	log('taskStatus: ' + taskStatus + ' counter: ' + counter);
	} else {
	log('Execute finish - ' + taskStatus);
	inProgress = false;
	}
}

log('Finished');

function log(msg) {
	document.write(msg );
	console.log(msg);
}

// ##### generic REST API calling method ##############
function callApi(path,data,token="",parseResult=true){
	var xhttp = new XMLHttpRequest();
	xhttp.open("POST", pyramidURL+path, false);
	xhttp.setRequestHeader("paToken",token)
	xhttp.send(JSON.stringify(data));
	if(parseResult){
		return JSON.parse(xhttp.responseText);
	}else{
		return xhttp.responseText;
	}
}

function readPieFile(file) {
	var rawFile = new XMLHttpRequest();
	rawFile.open('GET', file, false);
	rawFile.send();
	if (rawFile.readyState === 4 && rawFile.status === 200) {
		return rawFile.responseText;
	}
}